package com.tool_stdy.Spider;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.util.HashMap;
import java.util.Map.Entry;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

public class TUccSpider {

    private static final String BASEURL = "http://www.xiamp4.com";
    private static String Spiderurl = "http://www.xiamp4.com/GvodHtml/1_%s.html";
    static HttpClient httpClient = HttpClients.createDefault();
    static ConcurrentHashMap<String, String> resultMap = new ConcurrentHashMap<String, String>();
    private static volatile ArrayBlockingQueue<String> date = new ArrayBlockingQueue<String>(1000000);
    private static ExecutorService pool = Executors.newFixedThreadPool(50);
    private static CopyOnWriteArrayList<Future<String>> list = new CopyOnWriteArrayList<Future<String>>();
    private static volatile ArrayBlockingQueue<String> pageContent = new ArrayBlockingQueue<String>(1000000);
    public Matcher getMatcher(String targetSpider, String value) {
        Pattern compile = Pattern.compile(targetSpider);
        return compile.matcher(value);
    }

    public void getUrlContent(String httpUrl) {
        try {
            if(StringUtils.isBlank(httpUrl)){
                return;
            }
            HashMap<String, String> contentUrlName = new HashMap<String, String>();
            HttpGet get = new HttpGet(httpUrl);
            HttpResponse execute = httpClient.execute(get);
            String value = EntityUtils.toString(execute.getEntity(), "gbk");
            value = value.replaceAll("：", "/");
            String targetUrl = "<div class=\"info\"><h2><a href=\"/Html/\\w+.html\" title=\"([\u4e00-\u9fa5]|/|[\u4e00-\u9fa5])+\"";
            Matcher matcher = getMatcher(targetUrl, value);
            int i = 0;
            while (matcher.find()) {
                String urlResult = matcher.group();
                String contentUrl = urlResult.split("\"")[3];
                String name = urlResult.split("\"")[5];
                pageContent.add(name+"=="+contentUrl);
            }
            while(!pageContent.isEmpty()){
                String nameUrl = pageContent.poll();
                String shortUrl=nameUrl.split("==")[1];
                String moveName=nameUrl.split("==")[0];
                String contenturl = BASEURL +shortUrl;
                get = new HttpGet(contenturl);
                execute = httpClient.execute(get);
                value = EntityUtils.toString(execute.getEntity(), "gbk");
                String pageTargetUrl = "<a href=\"/Html/\\w+.html\" title=\"([\u4e00-\u9fa5]|/|[\u4e00-\u9fa5])+\"";
                Matcher ma= getMatcher(pageTargetUrl, value);//列表详情页里面同样含有点击入口
                while(ma.find()){
                    String urlResult = ma.group();
                    String contentUrl = urlResult.split("\"")[1];
                    String name = urlResult.split("\"")[3];
                    pageContent.add(name+"=="+contentUrl);
                }
                String target = "var GvodUrls = \"";
                String[] split = value.split(target);
                if(split.length<2){
                    continue;
                }
                String[] split2 = split[1].split("\";</script>");
                if(split2.length<2){
                    continue;
                }
                String ed2k = split2[0];
                resultMap.put(ed2k, moveName);
              System.out.println(Thread.currentThread().getName()+"==>"+moveName);
            }
            for (Entry<String, String> en : contentUrlName.entrySet()) {
            }

        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args){
        try {
            for (int i = 2; i < 146; i++) {
                String url = String.format(Spiderurl, String.valueOf(i));
                date.add(url);
            }
            while (!date.isEmpty()) {
                Future<String> submit = pool.submit(new Callable<String>() {
                    public String call() throws Exception {
                        TUccSpider tu = new TUccSpider();
                        tu.getUrlContent(date.poll());
                        return "success";
                    }
                });
                list.add(submit);
            }
            int j = 0;
            for (Future<String> f : list) {
                if (f != null && "success".equals(f.get())) {
                    j++;
                }
            }
            System.out.println("处理" + j + "条");
            File f = new File("d:/export");
            FileWriter fw = new FileWriter(f, true);
            BufferedWriter bf = new BufferedWriter(fw);
            for (Entry<String, String> en : resultMap.entrySet()) {
                bf.write(en.getValue() + ":" + en.getKey());
                bf.newLine();
            }
            bf.flush();
            bf.close();
            System.out.println(Thread.currentThread().getName() + "存储完毕!");
        } catch (Exception e) {
           e.printStackTrace();
        }
    }

}
